<?php 
	/**
	 * 自动注册有缘id
	 */
	function registerYouyuanId()
	{
		//step1: get zhenai view cookie
		$url = "http://www.youyuan.com/_register.html";
		$page = makeRequest($url,
						 "GET",
						 "",
						 true,
						 ROOT_PATH."/youyuan/cookie/youyuan_view_cookie.txt",
						 "",  
						 ""
						 );
		$exp = time() + 6*30*24*3600;
		$mictime = microtime(true)*100;
		$radomuid = $mictime.'_'.rand(10,99).'_'.rand(1,10);
		$text = 'www.youyuan.com'."\t".'TRUE'."\t".'/'."\t".'FALSE'."\t".$exp."\t".'radomUserId'."\t".$radomuid."\n";
		file_put_contents(ROOT_PATH."/youyuan/cookie/youyuan_view_cookie.txt",$text,FILE_APPEND);

		$text = 'www.youyuan.com'."\t".'TRUE'."\t".'/'."\t".'FALSE'."\t".$exp."\t".'out_url'."\t".'N'."\n";
		file_put_contents(ROOT_PATH."/youyuan/cookie/youyuan_view_cookie.txt",$text,FILE_APPEND);


		
		$url = "http://www.youyuan.com/user/reg.html";
		$mobile = makeRandMobile();
		$parameter = array("phoneNum" => $mobile,
							"password" => "123abc",
							"secondPassW" => "123abc",
							"agreeCheck" => "on",
							"age"=>23,
							"sex"=>0,
							"province"=>1,
							"city"=>0,
							"area"=>0,
							"username"=>$mobile,
							"password2"=>"123abc",
							);
		//var_dump($parameter);

		$opt_arr = array(CURLOPT_REFERER => "http://www.youyuan.com/_register.html",  //referer			
					);
		printlog("using:".$mobile." as register username", "register");
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true, 
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",  
						 ROOT_PATH."/Youyuan/cookie/youyuan_view_cookie.txt",
						 $opt_arr
						 );

		$url = 'http://www.youyuan.com/log.html';
		$user_time = intval($mictime /100);
		$parameter = array("t_user_id" => $radomuid,
							"t_user_time" => date('Y-m-d H:i:s',$user_time),
							"t_referrer" => "N",
							"referrer" => "/",
							"location"=>'/_register.html',
							"log_type"=>'A',
							"log_name"=>'REGISTER',
							"log_value"=>"N",
							);
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true, 
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",  
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",
						 $opt_arr
						 );
		$exp =  time() + 1*24*3600;
		$text = 'www.youyuan.com'."\t".'TRUE'."\t".'/'."\t".'FALSE'."\t".$exp."\t".'firstLoginFlg'."\t".'true'."\n";
		file_put_contents(ROOT_PATH."/youyuan/cookie/youyuan_login_cookie.txt",$text,FILE_APPEND);

		$url ='http://www.youyuan.com/space.html';
		$page = makeRequest($url,
					"GET",
					"",
					true,
					ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",  
					ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",
					""
				 );
		$url = 'http://www.youyuan.com/json/sign_in.html';
		
		$opt_arr = array(CURLOPT_REFERER => "http://www.youyuan.com/space.html",  //referer			
					);
		$parameter = array('');
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true,
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",  
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",
						 $opt_arr
						 );
		$url = 'http://www.youyuan.com/json/user.html';
		$parameter = array('');
		$page = makeRequest($url,
						 "POST",
						 $parameter,
						 true,
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",  
						 ROOT_PATH."/Youyuan/cookie/youyuan_login_cookie.txt",
						 $opt_arr
						 );
		$user = json_decode($page,TRUE);
		var_dump($user);
		return $user['guid'];
	}
	function makeRandMobile(){
		$prefix = array('134','135','136','137','138','139','150','151','152','157','158','159','147','182','183','187','188','130','131','132','155','156','185','186','145','133','','153','','180','','189');
		$rand_keys = array_rand($prefix, 1);
		$mobile = $prefix[$rand_keys].createRandNum();
		return $mobile;
	}
	function createRandNum($n = 8)
	{	
		$s = '';
		$str = "0123456789";   //   输出字符集  
		$len = strlen($str)-1;
		for($i=0 ; $i<$n; $i++){
			$s .=  $str[rand(0,$len)];  
		}
		
		return $s;
	}
	//自动保存新注册的id
	function saveYouyuanId($new_Youyuan_id)
	{
		global $conn;
		$today = date("Y-m-d", time());
		$now = date("Y-m-d H:i:s", time());
		$info = $now."\t".$new_Youyuan_id."\n";
		//保存至文件（防止数据库挂掉，备查）
		file_put_contents(ROOT_PATH."/youyuan/log/register_log/$today.log", $info, FILE_APPEND);
		//保存至数据库
		$arr = array("site" => "youyuan",
					"ymd" => $today,
					"register_time" => $now,
					"source_user_id" => $new_Youyuan_id
				);
		$sql = Sql::insert($arr, "register_log");
		echo $sql;
		$flag = $conn->query($sql);
		return $flag;
	}
	
	//获取某一天内有缘最小的id
	function getYouyuanMinId($ymd)
	{
		global $conn;
		$sql = "select min(source_user_id) from register_log where ymd='$ymd' and site='youyuan'";
		$minId = $conn->result($sql);
		return $minId;
	}
	
	//获取某一天内有缘最大的id
	function getYouyuanMaxId($ymd)
	{
		global $conn;
		$sql = "select max(source_user_id) from register_log where ymd='$ymd' and site='youyuan'";
		$maxId = $conn->result($sql);
		return $maxId;
	}
	
	//获取某一天内有缘已经抓到的最大id
	function getYouyuanMaxIdFinished($ymd,$middle,$ba=1)
	{
		global $conn;
		$tblName = getTblName($ymd);
		if ($ba == 1 ) {
			$sql = "select max(source_user_id) from $tblName where fetch_date='$ymd' and `from`='youyuan' and source_user_id<=$middle";//echo $sql;exit;
		}else{
			$sql = "select max(source_user_id) from $tblName where fetch_date='$ymd' and `from`='youyuan' and source_user_id>=$middle";//echo $sql;exit;
		}
		echo $sql."\n";
		
		$maxIdFinished = $conn->result($sql);
		return $maxIdFinished;
	}
	
	//获取有缘某一个id的注册时间，debug时候用
	function getYouyuanRegTime($uid)
	{
		global $conn;
		$sql = "select register_time from register_log where source_user_id='$uid' and site='youyuan'";
		$regTime = $conn->result($sql);
		return $regTime;
	}
	
	//判断有缘抓下来的页面是否包含正常的uid（通过经验判断，审核不通过的uid页面会非常小）
	function checkYouyuanIdExists($page)
	{
		if(strlen($page)<1024)
			return false;
		else
			return true;
	}
	function youyuan_data($startId, $endId, $ymd)
	{
		if($startId >= $endId){
			echo 'finish';
			exit;
		}
		//创建page目录
		$page_dir = ROOT_PATH."/youyuan/pages"."/".$ymd."/";
		if(!is_dir($page_dir)){
			echo 'ymd error ';
			//exit;
		}
		for ($uid=$startId; $uid<$endId; $uid++){
			$file = $page_dir.$uid.".html";
			$page = file_get_contents($file);
			doWithYouyuanPage($uid, $ymd, $page);
		}
		echo date('Y-m-d H:i:s'). 'end';
	}
	//主函数： 根据起始id，终止id 和 ymd 这几个值来处理珍爱业务，包括抓取、提取和入库
	function youyuan_crawl($startId, $endId, $ymd)
	{
		if($startId >= $endId){
			echo 'finish';
			exit;
		}
		//创建page目录
		$page_dir = ROOT_PATH."/youyuan/pages";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		
		$page_dir = $page_dir."/".$ymd."/";
		if(!is_dir($page_dir)){
			mkdir($page_dir);
		}
		//如果抓取的速度不够，改用multi
		$number = 30;//concurrent request number
		$continueMiss = 0;
		$previousExists = true;
		$previousId = $startId;
		for ($uid=$startId; $uid<$endId; ){
			//保留本次开始的uid，万一抓取失败了重新抓取
			$previousId = $uid; 
			$perRequestFailedCount  = 0 ;//记录本次抓取失败的个数
			$handles = makeYouyuanMultiRequests($uid, $number);
			foreach($handles as $thisuid => $handle){
				$page = curl_multi_getcontent($handle);
				if($page === false){ 
					printlog ("curl_multi_getcontent error: ".curl_error($handle), "crawl");
				}
				
				if(checkYouyuanIdExists($page)){
					printlog("$thisuid available", "crawl");
					if(file_put_contents($page_dir.$thisuid.".html", $page)){	
						//doWithYouyuanPage($thisuid, $ymd, $page);
					}
					else{
						printlog("save page error", "crawl");
					}
					$continueMiss = 0;
				}else{
					if(strlen($page)<30){
						echo "network problem ret is ".$page;
						$perRequestFailedCount++;
					}
					else{ 
						printlog("$thisuid not available", "crawl");
					}
					$continueMiss ++;
				}
				curl_close($handle);
				$uid++;
			}
				
			if($perRequestFailedCount>= $number/2){//如果有一半不存在，说明中途网络有问题了 
				$uid = $previousId;
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，重新来。。。", "crawl");
				continue;//重新来
			}
			else{
				printlog("本次并发有 $perRequestFailedCount 个抓取失败，继续下一次抓取", "crawl");
			}
		}
	} 
	/*
	*一下抓取多个页面
	*/
	function makeYouyuanMultiRequests($uid, $number)
	{
		$mhandle = curl_multi_init();
		$handles = array();
		while($number>0){
			$url = "http://www.youyuan.com/".$uid."-profile/";
			$opt_arr = array(CURLOPT_REFERER => "http://search.baidu.com/search",  //referer	
					);
			$handles[$uid] = makeRequestResource($url,
												"GET" , 
												"", 
												false, 
												"", 
												ROOT_PATH."/youyuan/cookie/youyuan_login_cookie.txt", 
												$opt_arr
												);
			$uid++;
			$number--;
		}
		foreach($handles as $handle)
			curl_multi_add_handle($mhandle, $handle);
		//wait all request finish
		do {
			$status = curl_multi_exec($mhandle, $active);
		} while ($status === CURLM_CALL_MULTI_PERFORM || $active);
		curl_multi_close($mhandle);
		
		return $handles;
	}
		
	//对一个抓下的有缘页面的后续处理
	function doWithYouyuanPage($uid, $ymd, $page)
	{
		//check uid exists?
		global $conn, $global_province_list;
		$tblName = getTblName($ymd);
		$sql = "select * from $tblName where `from`='youyuan' and source_user_id='$uid' limit 1";
		$exists = $conn->has_record($sql);
		if ($exists){
			printlog("$uid exists, do nothing", "crawl");
			return 1;
		}
		else{
			//extract
			//tmp: 山东济南34岁 或 江苏省20岁
			$page = str_replace(array(" ","\t","\n","\r","&nbsp;"), "", $page);
			//file_put_contents('aaab.txt', $page);

			preg_match('/<divclass="info">(.*?)<divid="bigPhotoDiv"/i', $page, $matches);
			$tmp = $matches[1];
			
			preg_match('%<title>(.*?)</title>%i', $page, $title);
			if(isset($title[1])){
				$ti =  $title[1];
			}else{
				$ti = '';
			}
			//$ti = iconv('utf-8', 'gbk', $ti);
			//echo $ti;
			//$tmp = iconv('gbk', 'utf-8', $tmp);
			preg_match('%收入：</label>(.*?)</dd>%i', $tmp,$income);
		
			if(!isset($income[1]) || $income[1] =='保密'){
				printlog("$uid is a fame user", "adata");
				return 1;
			}
			$temparr = explode('|', $ti);
			$province =$temparr[0];
			$city = $temparr[1];
			/*
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($ti, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			preg_match('/岁，在(.*?)，/i', $tmp, $city); //20
			$city = str_replace(array($province, "省"), "", strip_tags($city[1])); //济南
			$city = trim($city,'-');
			if ($city == "") {
				$city = '市区';
			}
			*/
						
			preg_match('/(\d\d)岁/', $tmp, $age); //20
			if (isset($age[1]) && $age[1]>0) {
				$age = $age[1];
			}else{
				$age = 18;
			}

			
			
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			if (strpos($ti, '男') !== false) {
				$gender = 1;
			}else{
				$gender = 0;
			}
			
			/*
			$gender = simpleExtract('/(.*)?士/', $tmp, "gender");
			$gender = ($gender=="男")?1:0; //男为1 女为0
			*/
			preg_match('%<ddclass="dd_photo">(.*?)</dd>%i', $tmp, $photo); //20
			//var_dump($photo);
			if (isset($photo[1]) && strpos($photo[1],'smallPhoto') !==false) {
				$has_photo = 1;
			}else{
				$has_photo = 0;
			}
			
			//insert
			$arr = array("fetch_date" => $ymd,
					"`from`" => "youyuan",
					"source_user_id" => $uid,
					"age" => $age,
					"province" => $province,
					"city" => $city,
					"gender" => $gender,
					"has_photo" => $has_photo,
				);
			//var_dump($arr);
			//exit;
			$sql = Sql::insert($arr, $tblName);
			$conn->query($sql);
		}
	}
	//对一个抓下的有缘页面的后续处理
	function doWithYouyuan($page)
	{
		//check uid exists?
		global $conn, $global_province_list;
			//extract
			//tmp: 山东济南34岁 或 江苏省20岁
			$tmp = simpleExtract('%profile_intro_row"><strong>(.*?)</strong>%i', $page, "tmp");
			$tmp = strip_tags($tmp);
			echo $tmp."<br />";
			
			$province = "";
			foreach ($global_province_list as $prov){
				if (strpos($tmp, $prov) !== false) {
					$province = $prov; //山东
					break;
				}
			}
			$age = simpleExtract('/(\d\d)/', $tmp, "age"); //20
			$city = simpleExtract('/来自(.*)?/', $tmp, "city"); //20
			$city = str_replace(array($province, $age, "省"), "", $city); //济南
			
			//echo "age: $age province: $province city: $city tmp: $tmp ";exit;
			$gender = simpleExtract('/(.*)?士/', $tmp, "gender");
			$gender = ($gender=="男")?1:0; //男为1 女为0
			
			preg_match('/<li.*?class="first-child".*?><img.*?src="(.*?)"/i', $page, $matches);
						
			$avat_array = array('300nv.png','300nj.png');
			if(isset($matches[1])){
				$photoarr = explode( '/' ,$matches[1]);
				if(is_array($photoarr)){
					$potofile = $photoarr[count($photoarr)-1];
					$has_photo = in_array($potofile,$avat_array)?0:1;
				}else{
					$has_photo = 0 ;
				}
			}else{
				preg_match('/<li.*?><img.*?name="profile_img".*?src="(.*?)"/i', $page, $match);
				if(isset($match[1])){
					$photoarray = explode( '/' ,$match[1]);
					if(is_array($photoarray)){
						$imgfile = $photoarray[count($photoarray)-1];
						$has_photo = in_array($imgfile ,$avat_array)?0:1;
					}else{
						$has_photo = 0 ;
					}
				}
			}
			echo "has_photo:".$has_photo."\n";
		
	}

?>